{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %load_ext autoreload\n",
    "# %autoreload 2\n",
    "# import sys; sys.path.append('/future/u/okhattab/repos/public/stanfordnlp/dspy')\n",
    "\n",
    "import dspy\n",
    "from dspy.evaluate import Evaluate\n",
    "from dspy.datasets.gsm8k import GSM8K, gsm8k_metric\n",
    "from dspy.teleprompt import BootstrapFewShotWithRandomSearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 7473/7473 [00:00<00:00, 39790.63it/s]\n",
      "100%|██████████| 1319/1319 [00:00<00:00, 40274.36it/s]\n"
     ]
    }
   ],
   "source": [
    "gms8k = GSM8K()\n",
    "turbo = dspy.OpenAI(model='gpt-3.5-turbo-instruct', max_tokens=250)\n",
    "\n",
    "trainset, devset = gms8k.train, gms8k.dev\n",
    "\n",
    "dspy.settings.configure(lm=turbo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_THREADS = 4\n",
    "evaluate = Evaluate(devset=devset[:], metric=gsm8k_metric, num_threads=NUM_THREADS, display_progress=True, display_table=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CoT(dspy.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.prog = dspy.ChainOfThought(\"question -> answer\")\n",
    "    \n",
    "    def forward(self, question):\n",
    "        return self.prog(question=question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "RUN_FROM_SCRATCH = False\n",
    "\n",
    "if RUN_FROM_SCRATCH:\n",
    "    config = dict(max_bootstrapped_demos=8, max_labeled_demos=8, num_candidate_programs=10, num_threads=NUM_THREADS)\n",
    "    teleprompter = BootstrapFewShotWithRandomSearch(metric=gsm8k_metric, **config)\n",
    "    cot_bs = teleprompter.compile(CoT(), trainset=trainset, valset=devset)\n",
    "    # cot_bs.save('turbo_8_8_10_gsm8k_200_300.json')\n",
    "else:\n",
    "    cot_bs = CoT()\n",
    "    cot_bs.load('turbo_8_8_10_gsm8k_200_300.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 241 / 300  (80.3): 100%|██████████| 300/300 [00:00<00:00, 433.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Metric: 241 / 300  (80.3%)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "80.33"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(cot_bs, devset=devset[:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "Given the fields `question`, produce the fields `answer`.\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Question: ${question}\n",
      "Reasoning: Let's think step by step in order to ${produce the answer}. We ...\n",
      "Answer: ${answer}\n",
      "\n",
      "---\n",
      "\n",
      "Question: Mark is baking bread. He has to let it rise for 120 minutes twice. He also needs to spend 10 minutes kneading it and 30 minutes baking it. How many minutes does it take Mark to finish making the bread?\n",
      "Reasoning: Let's think step by step in order to find the total time it takes Mark to finish making the bread. We know that he has to let the bread rise for 120 minutes twice, which is 240 minutes. He also needs to spend 10 minutes kneading it and 30 minutes baking it. Therefore, the total time it takes Mark to finish making the bread is 240 + 10 + 30 = 280 minutes.\n",
      "Answer: 280\n",
      "\n",
      "---\n",
      "\n",
      "Question: Ben has $2000 for his business operations costs. He orders goods from his supplier and writes them a cheque for $600. His debtor pays him $800 from the purchases they had made on credit. Mr. Ben then decides to do equipment maintenance and spends $1200 on the whole operation. How much money is Mr. Ben remaining with?\n",
      "Reasoning: Let's think step by step in order to find the remaining amount of money. We first start with the total amount of money Mr. Ben has, which is $2000. Then, we subtract the amount he spent on goods from his supplier, which is $600. Next, we add the amount he received from his debtor, which is $800. Finally, we subtract the amount he spent on equipment maintenance, which is $1200. We can represent this as $2000 - $600 + $800 - $1200 = $1000.\n",
      "Answer: 1000\n",
      "\n",
      "---\n",
      "\n",
      "Question: Debbie works at a post office packing boxes to mail. Each large box takes 4 feet of packing tape to seal, each medium box takes 2 feet of packing tape to seal, and each small box takes 1 foot of packing tape to seal. Each box also takes 1 foot of packing tape to stick the address label on. Debbie packed two large boxes, eight medium boxes, and five small boxes this afternoon. How much tape did she use?\n",
      "Reasoning: Let's think step by step in order to find the total amount of tape Debbie used. We know that she used 4 feet of tape for each large box, 2 feet of tape for each medium box, and 1 foot of tape for each small box. Therefore, we can calculate the total amount of tape used for each type of box by multiplying the number of boxes by the amount of tape needed for each box. For the large boxes, she used 4 feet of tape for each of the 2 boxes, so she used 8 feet of tape. For the medium boxes, she used 2 feet of tape for each of the 8 boxes, so she used 16 feet of tape. For the small boxes, she used 1 foot of tape for each of the 5 boxes, so she used 5 feet of tape. In total, she used 8 + 16 + 5 = 29 feet of tape. Additionally, she used 1 foot of tape for each box to stick the address label, so she used 15 feet of tape for the 15 boxes. Therefore, the total amount of tape she used is 29 + 15 = 44 feet.\n",
      "Answer: 44 feet\n",
      "\n",
      "---\n",
      "\n",
      "Question: Heloise has dogs and cats in the ratio of 10:17, with the total number of pets being 189. If she gives 10 dogs to her friend Janet, how many dogs does she remain with altogether?\n",
      "Reasoning: Let's think step by step in order to find the number of dogs Heloise remains with. We know that the total number of pets is 189, and the ratio of dogs to cats is 10:17. This means that for every 10 dogs, there are 17 cats. If we divide 189 by 27 (10+17), we get 7. This means that for every 7 sets of 10 dogs and 17 cats, there are 189 pets. Since we are only interested in the number of dogs, we can multiply 7 by 10 to get 70 dogs. If Heloise gives 10 dogs to Janet, she will remain with 70-10 = 60 dogs.\n",
      "Answer: 60\n",
      "\n",
      "---\n",
      "\n",
      "Question: Jorge is 24 years younger than Simon. In 2005, Jorge is 16 years old. In 2010, how old would Simon be?\n",
      "Answer: 45\n",
      "\n",
      "---\n",
      "\n",
      "Question: Sally is selling boxes of crackers for her scout troop's fund-raiser. If she sells 50% more on Sunday than she sold on Saturday, then she'll have sold a total of 150 boxes on the two days. How many boxes did she sell on Saturday?\n",
      "Answer: 60\n",
      "\n",
      "---\n",
      "\n",
      "Question: Audrey is 7 years older than Heracles. In 3 years, Audrey will be twice as old as Heracles is now. How old is Heracles now?\n",
      "Answer: 10\n",
      "\n",
      "---\n",
      "\n",
      "Question: Billy is breeding mice for an experiment. He starts with 8 mice, who each have 6 pups. When the pups grow up, all the mice have another 6 pups. Then each adult mouse eats 2 of their pups due to the stress of overcrowding. How many mice are left?\n",
      "Answer: 280\n",
      "\n",
      "---\n",
      "\n",
      "Question: Cloud 9 Diving Company has taken individual bookings worth $12,000 and group bookings worth $16,000. Some people have cancelled at the last minute. $1600 has had to be returned to them. How much money has the sky diving company taken altogether?\n",
      "Reasoning: Let's think step by step in order to\u001b[32m find the total amount of money the sky diving company has taken. We know that they have taken individual bookings worth $12,000 and group bookings worth $16,000, for a total of $28,000. However, they had to return $1600 to some people who cancelled at the last minute. Therefore, the total amount of money they have taken is $28,000 - $1600 = $26,400.\n",
      "Answer: 26400\u001b[0m\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "turbo.inspect_history(n=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py39_aug2023_dspy",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.17"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
